#!/bin/bash
export TRAIN_FILE=/data/paraphrase/wikianswers/all_text.txt
export CLS_MODEL=/data/paraphrase/models/paraphrase-cls
export MODEL=/data/title_generation/models/unilm_base_uncased_v1.2/pytorch_model.bin
export OUTPUT_DIR=/data/paraphrase/models/wiki_lm_att
export CACHE_DIR=/data/paraphrase/cache_att
export CONFIG_PATH=/data/paraphrase/models/wiki_lm/config.json
export CUDA_VISIBLE_DEVICES=0,1,2,3

python run_language_modeling_att.py \
    --output_dir $OUTPUT_DIR \
    --model_type bert  \
    --model_name_or_path $MODEL --cls_model_name_or_path $CLS_MODEL --cache_dir $CACHE_DIR \
    --do_train \
    --train_data_file=$TRAIN_FILE --per_gpu_train_batch_size 48 \
    --mlm --line_by_line --config_name $CONFIG_PATH --tokenizer_name bert-base-uncased
